﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Net;
using HtmlAgilityPack;

namespace Spider
{
    public class Spider
    {
        private void SetInternalLinks(string html, CrawledData craweledData)
        {
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(html);

            var aboutUsLink = document.DocumentNode.SelectNodes("//a[contains(translate(@href,'ABCDEFGHIJKLOMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'about')]/@href");
            var contactLink = document.DocumentNode.SelectNodes("//a[contains(translate(@href,'ABCDEFGHIJKLOMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'contact')]/@href");
            var pinItButton = document.DocumentNode.SelectNodes("//img[contains(translate(@src,'ABCDEFGHIJKLOMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'//assets.pinterest.com/images/')]");
            craweledData.AboutUrl = aboutUsLink != null && aboutUsLink.Count > 0 ? aboutUsLink.ElementAt(0).Attributes["href"].Value : string.Empty;
            craweledData.ContactUrl = contactLink != null && contactLink.Count > 0 ? contactLink.ElementAt(0).Attributes["href"].Value : string.Empty;
            craweledData.DoesIncludePinItButton = pinItButton != null;
        }

        public CrawledData FindPageInfo(string url)
        {
            var craweledData = new CrawledData();
            craweledData.OriginalUrl = url;
            var html = DownloadHtml(url);
            SetInternalLinks(html, craweledData);
            SetExternalLinks(html, craweledData);
            return craweledData;
        }

        public IEnumerable<string> FindAllEmails(CrawledData data)
        {
            var contactEmails = FindEmails(data.ConvertToAbsoulteUrl(data.ContactUrl));
            var aboutUsEmails = FindEmails(data.ConvertToAbsoulteUrl(data.AboutUrl));
            var facebookEmails = FindEmails(data.FacebokUrl);

            contactEmails.AddRange(aboutUsEmails);
            contactEmails.AddRange(facebookEmails);
            contactEmails.Distinct();
            return contactEmails;
        }

        public List<string> FindEmails(string url)
        {
            if (string.IsNullOrEmpty(url) || !Uri.IsWellFormedUriString(url, UriKind.Absolute))
                return new List<string>();

            using (var client = new WebClient())
            {
                try
                {
                    client.Headers.Add("User-Agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");
                    var html = client.DownloadString(url);
                    var replacedContent = html.Replace("&#064;", "@");
                    return ExtractEmail(replacedContent);
                }
                catch
                {
                    //TODO: log error
                    return new List<string>();
                }
            }
        }

        private List<string> ExtractEmail(string text)
        {
            //replacing strange characters that are usually appended to @ with empty spaces
            var emailText = text.Replace("", "");
            emailText = emailText.Replace("*", "");
            emailText = emailText.Replace("#", "");

            const string matchEmailPattern =
            @"(([\w-]+\.)+[\w-]+|([a-zA-Z]{1}|[\w-]{2,}))@"
            + @"((([0-1]?[0-9]{1,2}|25[0-5]|2[0-4][0-9])\.([0-1]?[0-9]{1,2}|25[0-5]|2[0-4][0-9])\."
            + @"([0-1]?[0-9]{1,2}|25[0-5]|2[0-4][0-9])\.([0-1]?[0-9]{1,2}|25[0-5]|2[0-4][0-9])){1}|"
            + @"([a-zA-Z]+[\w-]+\.)+[a-zA-Z]{2,4})";

            var rx = new Regex(matchEmailPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
            // Find matches.
            var matches = rx.Matches(emailText);
            // Report on each match.
            return (from Match match in matches select match.Value).ToList();
        }

        private void SetExternalLinks(string html, CrawledData craweledData)
        {
            craweledData.FacebokUrl = GetFacebookLink(html);
            craweledData.PinterestlUrl = GetPinterestLink(html);
        }

        private string GetFacebookLink(string html)
        {
            //taken from - http://stackoverflow.com/questions/5205652/facebook-profile-url-regular-expression
            var regex = @"(?:http:\/\/)?(?:www.)?facebook.com\/(?:(?:\w)*#!\/)?(?:pages\/)?(?:[?\w\-]*\/)?(?:profile.php\?id=(?=\d.*))?([\w\-]*)?";
            var rx = new Regex(regex, RegexOptions.Compiled | RegexOptions.IgnoreCase);
            // Find matches.
            var matches = rx.Matches(html);
            // Report on each match.
            return (from Match match in matches select match.Value).FirstOrDefault(match => !DoesContainIrrelevantFbTags(match));
        }

    
        private bool DoesContainIrrelevantFbTags(string link)
        {
            var lLink = link.ToLower();
            return lLink.Contains("/plugin") || lLink.Contains("/fbml") 
                || lLink.Contains("/sharer") || lLink.Contains("/widget") ;
        }

        private string GetPinterestLink(string html)
        {
            var regex = @"(?:http:\/\/)?(?:www.)?pinterest.com/?([\w\-]*)?";
            var rx = new Regex(regex, RegexOptions.Compiled | RegexOptions.IgnoreCase);
            // Find matches.
            var matches = rx.Matches(html);
            // Report on each match.
            return (from Match match in matches select match.Value).FirstOrDefault();
        }

        private string DownloadHtml(string url)
        {
            if (string.IsNullOrEmpty(url) || !Uri.IsWellFormedUriString(url, UriKind.Absolute))
                return string.Empty;
            using (var client = new WebClient())
            {
                try
                {
                    client.Headers.Add("User-Agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");
                    return client.DownloadString(url);
                }
                catch
                {
                    //TODO: Log exception
                    return string.Empty;
                }
            }
        }
    }
}
